import fasttext
import numpy
import shap 
from lime import lime_text

from lime.lime_text import LimeTextExplainer
explainer = LimeTextExplainer(kernel_width=25, class_names=class_names)
exp = explainer.explain_instance(sentences[8], predict_probability, num_features=10, labels=[0,2])
exp.show_in_notebook()

def predict_probability(sentences):
    order = ["__label__tam", "__label__kan", "__label__mal", "__label__tel"]
    output = [] 
    for s in sentences:
        prediction = model.predict(s,k=-1)
        zipped = dict(zip(prediction[0],prediction[1]) )
        ordered = numpy.array([zipped[x] for x in order])
        output.append(ordered)
    overall =numpy.array(output)
    # print(overall.shape)
    return  overall
    
DESTPATH = "../labelled_data/Flatten/flatten_all_"
SOURCEPATH = "../labelled_data/"

scripts = ["Taml", "Mlym", "Knda", "Telu"]
langs = [("Tamil", "tam"), ("Telugu", "tel"), ("Kannada", "kan"), ("Malayalam", "mal")]

script = scripts[3]
with open(DESTPATH + script + ".dev", "r") as f:
   data = [x.strip() for x in f.readlines()]
   
model = fasttext.load_model(f"../models/flatten_model_{script}.bin")
class_names = [ "Tamil","Kannada","Malayalam", "Telugu"]

Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.

sentences = []
differences = [] 
i = 0
predicted_labels = []
second_labels = []
gold_labels = []
mask = []
while True:
    label = data[i].split(" ")[0]
    sentence = " ".join(data[i].split(" ")[1:])
    prediction = model.predict(sentence, k=2)
    differences.append(abs(prediction[1][0] - prediction[1][1]))
    sentences.append(sentence)
    predicted_labels.append(prediction[0][0])
    second_labels.append(prediction[0][1])
    gold_labels.append(label)
        
    i = i + 1
    if i == len(data):
        break
   
        
import matplotlib.pyplot as plt
from matplotlib import font_manager as fm

fe = fm.FontEntry(
    fname='../venv/Noto_Sans_Telugu/NotoSansTelugu-VariableFont_wdth,wght.ttf',
    name='Noto_Sans_Telugu')
fm.fontManager.ttflist.insert(0, fe) # or append is fine
plt.rcParams['font.family'] = fe.name # = 'your custom ttf font name'

import numpy
import matplotlib
maskapply = numpy.asarray(predicted_labels) != "__label__tam"
difference_array = numpy.asarray(differences)[maskapply] 
sentence_array = numpy.asarray(sentences)[maskapply] 
predicted_labels = numpy.asarray(predicted_labels)[maskapply]
gold_labels = numpy.asarray(gold_labels)[maskapply] 
ix_array = numpy.asarray(list(range(0,len(sentences))))[maskapply] 


maskapply = gold_labels == "__label__tam"
difference_array = difference_array[maskapply]
sentence_array = sentence_array[maskapply]
gold_labels = gold_labels[maskapply]
predicted_labels = predicted_labels[maskapply]
ix_array = ix_array[maskapply]
plt.hist(sorted(difference_array))

maskapply = difference_array < 0.05
difference_array_05 = difference_array[maskapply]
sentence_array_05 = sentence_array[maskapply]
gold_labels_05 = gold_labels[maskapply]
predicted_labels_05 = predicted_labels[maskapply]
ix_array_05 = ix_array[maskapply]

maskapply = difference_array > 0.15
difference_array = difference_array[maskapply]
sentence_array = sentence_array[maskapply]
gold_labels = gold_labels[maskapply]
predicted_labels = predicted_labels[maskapply]
ix_array = ix_array[maskapply]
plt.hist(sorted(difference_array))
print(len(difference_array))
# For when the prediction is NOT-TAMIL, the prediction is usually not confident. These are about 100 
# sentences out of 1000, i.e. 10%. Out of them 50 are such where the difference between the top 2 predictions is
# less than 0.05 i.e 5%. We will analyze these
print(sorted(difference_array))

9
[0.15131700038909912, 0.15222442150115967, 0.15474256873130798, 0.15691295266151428, 0.15919628739356995, 0.17793524265289307, 0.1802663505077362, 0.20138287544250488, 0.32773756980895996]

from collections import Counter 
Counter(predicted_labels)

Counter({'__label__tel': 981,
         '__label__mal': 1114,
         '__label__kan': 997,
         '__label__tam': 896})

masker = shap.maskers.Text(r'\W+')  # this will create a basic whitespace tokenizer
shap_explainer = shap.Explainer(predict_probability, masker, output_names=["__label__tam", "__label__kan", "__label__mal", "__label__tel"])
shap_values = shap_explainer(sentences)

PartitionExplainer explainer: 3989it [13:09,  4.99it/s]

import random 
ix_array_05_random = random.sample(set(ix_array_05), 10)

for i in ix_array_05_random:
    shap.plots.text(shap_values[i])

ix_array

array([ 906, 1113, 1395, 1687, 2080, 2108, 2224, 2270, 2801])

for i in ix_array:
    shap.plots.text(shap_values[i])

for i in ix_array:
    shap.plots.bar(shap_values[i][:,2], max_display=20)

masker = shap.maskers.Text(r' ')  # this will create a basic whitespace tokenizer
shap_explainer = shap.Explainer(predict_probability, masker, output_names=["__label__tam", "__label__kan", "__label__mal", "__label__tel"])
shap_values = shap_explainer(sentences)

PartitionExplainer explainer: 3989it [05:03, 12.74it/s]

for i in ix_array_05_random:
    shap.plots.text(shap_values[i])

for i in ix_array_05_random:
    shap.plots.bar(shap_values[i][:,2], max_display=20)

for i in ix_array:
    shap.plots.text(shap_values[i])

for i in ix_array:
    shap.plots.bar(shap_values[i][:,2], max_display=20)

for i in ix_array:
    shap.plots.bar(shap_values[i][:,0], max_display=20)

Character Level Explanations (< 0.05)¶

Character Level Explanations > 0.15¶

Word Level Explanations¶

Explanations < 0.05¶

Explanations > 0.15¶